package com.arelikebrothers.lottery.data.core.spyder;

import com.arelikebrothers.lottery.data.core.entity.Province;
import lombok.Getter;
import lombok.Setter;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * Created by lennon on 12/08/2017.
 */
public abstract class AbstractPageProcessor implements PageProcessor, Runnable {


    @Getter
    @Setter
    private Request[] requests;

    // 获取爬取的网址
    public abstract String[] getUrls();

    // 设置pipeline
    public abstract Pipeline getPipeLine();

    @Override
    public void run() {
        Spider spider = Spider.create(this)
                .addPipeline(getPipeLine())
                .addUrl(getUrls())
                .thread(8);
        if (requests != null) {
            spider.addRequest(requests);
        }
        spider.run();
    }

    public Site getSite() {
        Site site = Site.me()
                .setRetryTimes(3000)
                .setTimeOut(30000)
                .setSleepTime(50);
        return site;
    }

    public abstract Province getProvince();
}
